{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "R619Z2Qy7IrX"
   },
   "outputs": [],
   "source": [
    "#挂载到谷歌云盘，本地跑不用挂载\n",
    "from google.colab import drive\n",
    "drive.mount('/content/drive')\n",
    "!pip install keras-transformer\n",
    "!pwd\n",
    "!ls \"/content/drive/My Drive/Colab Notebooks/middle_data/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3171,
     "status": "ok",
     "timestamp": 1588303923502,
     "user": {
      "displayName": "Jayee Wong",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhdF_nXQgSlUtrprqDLXzf9Kn59RlonvtGF4nHZ=s64",
      "userId": "08268369137892915441"
     },
     "user_tz": -480
    },
    "id": "2PLmOwOK6Toh",
    "outputId": "dad250e0-757c-4d06-d9ff-697bde7150bd"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pickle\n",
    "import operator\n",
    "from keras_transformer import get_model, decode\n",
    "# main_path = '/content/drive/My Drive/Colab Notebooks/'    #Google Colab FilePath\n",
    "# path = main_path + 'middle_data/'\n",
    "path = 'middle_data/'\n",
    "with open(path + 'encode_input.pkl', 'rb') as f:\n",
    "    encode_input = pickle.load(f)\n",
    "with open(path + 'decode_input.pkl', 'rb') as f:\n",
    "    decode_input = pickle.load(f)\n",
    "with open(path + 'decode_output.pkl', 'rb') as f:\n",
    "    decode_output = pickle.load(f)\n",
    "with open(path + 'source_token_dict.pkl', 'rb') as f:\n",
    "    source_token_dict = pickle.load(f)\n",
    "with open(path + 'target_token_dict.pkl', 'rb') as f:\n",
    "    target_token_dict = pickle.load(f)\n",
    "with open(path + 'source_tokens.pkl', 'rb') as f:\n",
    "    source_tokens = pickle.load(f)\n",
    "print('Done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 84
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2752,
     "status": "ok",
     "timestamp": 1588303929770,
     "user": {
      "displayName": "Jayee Wong",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhdF_nXQgSlUtrprqDLXzf9Kn59RlonvtGF4nHZ=s64",
      "userId": "08268369137892915441"
     },
     "user_tz": -480
    },
    "id": "em3y9E2S6Too",
    "outputId": "71c49f8b-a654-46c4-d381-76495a22b272"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10565\n",
      "7307\n",
      "20403\n",
      "WARNING:tensorflow:From c:\\users\\王嘉意\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\tensorflow\\python\\framework\\op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From C:\\Users\\王嘉意\\AppData\\Roaming\\Python\\Python36\\site-packages\\keras\\backend\\tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "Done\n"
     ]
    }
   ],
   "source": [
    "print(len(source_token_dict))\n",
    "print(len(target_token_dict))\n",
    "print(len(encode_input))\n",
    "# 构建模型\n",
    "model = get_model(\n",
    "    token_num=max(len(source_token_dict), len(target_token_dict)),\n",
    "    embed_dim=64,\n",
    "    encoder_num=2,\n",
    "    decoder_num=2,\n",
    "    head_num=4,\n",
    "    hidden_dim=256,\n",
    "    dropout_rate=0.05,\n",
    "    use_same_embed=False,  # 不同语言需要使用不同的词嵌入\n",
    ")\n",
    "model.compile('adam', 'sparse_categorical_crossentropy')\n",
    "# model.summary()\n",
    "print('Done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "collapsed": true,
    "executionInfo": {
     "elapsed": 2704582,
     "status": "ok",
     "timestamp": 1588306654812,
     "user": {
      "displayName": "Jayee Wong",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhdF_nXQgSlUtrprqDLXzf9Kn59RlonvtGF4nHZ=s64",
      "userId": "08268369137892915441"
     },
     "user_tz": -480
    },
    "id": "bezIJiRX6Tox",
    "outputId": "a9eda80b-c48b-402d-f43f-72252a193f5d"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/indexed_slices.py:434: UserWarning: Converting sparse IndexedSlices to a dense Tensor of unknown shape. This may consume a large amount of memory.\n",
      "  \"Converting sparse IndexedSlices to a dense Tensor of unknown shape. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "20000/20000 [==============================] - 29s 1ms/step - loss: 1.2761\n",
      "Epoch 2/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.9421\n",
      "\n",
      "Epoch 00002: loss improved from inf to 0.94212, saving model to /content/drive/My Drive/Colab Notebooks/modles/W--  2-0.9421-.h5\n",
      "Epoch 3/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.8060\n",
      "Epoch 4/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.7061\n",
      "\n",
      "Epoch 00004: loss improved from 0.94212 to 0.70608, saving model to /content/drive/My Drive/Colab Notebooks/modles/W--  4-0.7061-.h5\n",
      "Epoch 5/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.6181\n",
      "Epoch 6/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.5381\n",
      "\n",
      "Epoch 00006: loss improved from 0.70608 to 0.53811, saving model to /content/drive/My Drive/Colab Notebooks/modles/W--  6-0.5381-.h5\n",
      "Epoch 7/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.4686\n",
      "Epoch 8/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.4089\n",
      "\n",
      "Epoch 00008: loss improved from 0.53811 to 0.40895, saving model to /content/drive/My Drive/Colab Notebooks/modles/W--  8-0.4089-.h5\n",
      "Epoch 9/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.3579\n",
      "Epoch 10/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.3135\n",
      "\n",
      "Epoch 00010: loss improved from 0.40895 to 0.31351, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 10-0.3135-.h5\n",
      "Epoch 11/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.2782\n",
      "Epoch 12/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.2473\n",
      "\n",
      "Epoch 00012: loss improved from 0.31351 to 0.24731, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 12-0.2473-.h5\n",
      "Epoch 13/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.2192\n",
      "Epoch 14/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.1973\n",
      "\n",
      "Epoch 00014: loss improved from 0.24731 to 0.19733, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 14-0.1973-.h5\n",
      "Epoch 15/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.1807\n",
      "Epoch 16/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.1663\n",
      "\n",
      "Epoch 00016: loss improved from 0.19733 to 0.16630, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 16-0.1663-.h5\n",
      "Epoch 17/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.1534\n",
      "Epoch 18/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.1424\n",
      "\n",
      "Epoch 00018: loss improved from 0.16630 to 0.14236, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 18-0.1424-.h5\n",
      "Epoch 19/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.1327\n",
      "Epoch 20/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.1252\n",
      "\n",
      "Epoch 00020: loss improved from 0.14236 to 0.12522, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 20-0.1252-.h5\n",
      "Epoch 21/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.1185\n",
      "Epoch 22/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.1109\n",
      "\n",
      "Epoch 00022: loss improved from 0.12522 to 0.11087, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 22-0.1109-.h5\n",
      "Epoch 23/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.1053\n",
      "Epoch 24/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.1019\n",
      "\n",
      "Epoch 00024: loss improved from 0.11087 to 0.10189, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 24-0.1019-.h5\n",
      "Epoch 25/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0956\n",
      "Epoch 26/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0908\n",
      "\n",
      "Epoch 00026: loss improved from 0.10189 to 0.09081, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 26-0.0908-.h5\n",
      "Epoch 27/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0864\n",
      "Epoch 28/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0828\n",
      "\n",
      "Epoch 00028: loss improved from 0.09081 to 0.08283, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 28-0.0828-.h5\n",
      "Epoch 29/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0809\n",
      "Epoch 30/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0772\n",
      "\n",
      "Epoch 00030: loss improved from 0.08283 to 0.07720, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 30-0.0772-.h5\n",
      "Epoch 31/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0746\n",
      "Epoch 32/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0711\n",
      "\n",
      "Epoch 00032: loss improved from 0.07720 to 0.07106, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 32-0.0711-.h5\n",
      "Epoch 33/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0697\n",
      "Epoch 34/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0676\n",
      "\n",
      "Epoch 00034: loss improved from 0.07106 to 0.06762, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 34-0.0676-.h5\n",
      "Epoch 35/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0646\n",
      "Epoch 36/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0622\n",
      "\n",
      "Epoch 00036: loss improved from 0.06762 to 0.06223, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 36-0.0622-.h5\n",
      "Epoch 37/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0604\n",
      "Epoch 38/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0598\n",
      "\n",
      "Epoch 00038: loss improved from 0.06223 to 0.05976, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 38-0.0598-.h5\n",
      "Epoch 39/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0574\n",
      "Epoch 40/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0563\n",
      "\n",
      "Epoch 00040: loss improved from 0.05976 to 0.05630, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 40-0.0563-.h5\n",
      "Epoch 41/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0549\n",
      "Epoch 42/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0541\n",
      "\n",
      "Epoch 00042: loss improved from 0.05630 to 0.05411, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 42-0.0541-.h5\n",
      "Epoch 43/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0525\n",
      "Epoch 44/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0513\n",
      "\n",
      "Epoch 00044: loss improved from 0.05411 to 0.05132, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 44-0.0513-.h5\n",
      "Epoch 45/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0499\n",
      "Epoch 46/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0487\n",
      "\n",
      "Epoch 00046: loss improved from 0.05132 to 0.04874, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 46-0.0487-.h5\n",
      "Epoch 47/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0476\n",
      "Epoch 48/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0455\n",
      "\n",
      "Epoch 00048: loss improved from 0.04874 to 0.04545, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 48-0.0455-.h5\n",
      "Epoch 49/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0453\n",
      "Epoch 50/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0449\n",
      "\n",
      "Epoch 00050: loss improved from 0.04545 to 0.04488, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 50-0.0449-.h5\n",
      "Epoch 51/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0440\n",
      "Epoch 52/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0433\n",
      "\n",
      "Epoch 00052: loss improved from 0.04488 to 0.04332, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 52-0.0433-.h5\n",
      "Epoch 53/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0416\n",
      "Epoch 54/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0415\n",
      "\n",
      "Epoch 00054: loss improved from 0.04332 to 0.04147, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 54-0.0415-.h5\n",
      "Epoch 55/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0406\n",
      "Epoch 56/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0405\n",
      "\n",
      "Epoch 00056: loss improved from 0.04147 to 0.04050, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 56-0.0405-.h5\n",
      "Epoch 57/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0396\n",
      "Epoch 58/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0383\n",
      "\n",
      "Epoch 00058: loss improved from 0.04050 to 0.03826, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 58-0.0383-.h5\n",
      "Epoch 59/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0378\n",
      "Epoch 60/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0379\n",
      "\n",
      "Epoch 00060: loss improved from 0.03826 to 0.03793, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 60-0.0379-.h5\n",
      "Epoch 61/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0379\n",
      "\n",
      "Epoch 00061: ReduceLROnPlateau reducing learning rate to 0.00020000000949949026.\n",
      "Epoch 62/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0220\n",
      "\n",
      "Epoch 00062: loss improved from 0.03793 to 0.02205, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 62-0.0220-.h5\n",
      "Epoch 63/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0159\n",
      "Epoch 64/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0142\n",
      "\n",
      "Epoch 00064: loss improved from 0.02205 to 0.01418, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 64-0.0142-.h5\n",
      "Epoch 65/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0130\n",
      "Epoch 66/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0123\n",
      "\n",
      "Epoch 00066: loss improved from 0.01418 to 0.01234, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 66-0.0123-.h5\n",
      "Epoch 67/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0120\n",
      "Epoch 68/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0116\n",
      "\n",
      "Epoch 00068: loss improved from 0.01234 to 0.01164, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 68-0.0116-.h5\n",
      "Epoch 69/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0115\n",
      "Epoch 70/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0114\n",
      "\n",
      "Epoch 00070: loss improved from 0.01164 to 0.01139, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 70-0.0114-.h5\n",
      "Epoch 71/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0109\n",
      "Epoch 72/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0107\n",
      "\n",
      "Epoch 00072: loss improved from 0.01139 to 0.01070, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 72-0.0107-.h5\n",
      "Epoch 73/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0109\n",
      "Epoch 74/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0103\n",
      "\n",
      "Epoch 00074: loss improved from 0.01070 to 0.01031, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 74-0.0103-.h5\n",
      "Epoch 75/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0102\n",
      "Epoch 76/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0103\n",
      "\n",
      "Epoch 00076: loss improved from 0.01031 to 0.01030, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 76-0.0103-.h5\n",
      "Epoch 77/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0101\n",
      "\n",
      "Epoch 00077: ReduceLROnPlateau reducing learning rate to 4.0000001899898055e-05.\n",
      "Epoch 78/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0085\n",
      "\n",
      "Epoch 00078: loss improved from 0.01030 to 0.00848, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 78-0.0085-.h5\n",
      "Epoch 79/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0080\n",
      "Epoch 80/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0077\n",
      "\n",
      "Epoch 00080: loss improved from 0.00848 to 0.00768, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 80-0.0077-.h5\n",
      "Epoch 81/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0074\n",
      "Epoch 82/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0074\n",
      "\n",
      "Epoch 00082: loss improved from 0.00768 to 0.00738, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 82-0.0074-.h5\n",
      "Epoch 83/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0072\n",
      "Epoch 84/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0072\n",
      "\n",
      "Epoch 00084: loss improved from 0.00738 to 0.00717, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 84-0.0072-.h5\n",
      "Epoch 85/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0071\n",
      "Epoch 86/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0070\n",
      "\n",
      "Epoch 00086: loss improved from 0.00717 to 0.00704, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 86-0.0070-.h5\n",
      "Epoch 87/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0070\n",
      "Epoch 88/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0071\n",
      "\n",
      "Epoch 00088: loss did not improve from 0.00704\n",
      "\n",
      "Epoch 00088: ReduceLROnPlateau reducing learning rate to 8.000000525498762e-06.\n",
      "Epoch 89/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0067\n",
      "Epoch 90/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0067\n",
      "\n",
      "Epoch 00090: loss improved from 0.00704 to 0.00666, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 90-0.0067-.h5\n",
      "Epoch 91/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0066\n",
      "Epoch 92/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0067\n",
      "\n",
      "Epoch 00092: loss did not improve from 0.00666\n",
      "Epoch 93/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0067\n",
      "\n",
      "Epoch 00093: ReduceLROnPlateau reducing learning rate to 1.6000001778593287e-06.\n",
      "Epoch 94/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0064\n",
      "\n",
      "Epoch 00094: loss improved from 0.00666 to 0.00636, saving model to /content/drive/My Drive/Colab Notebooks/modles/W-- 94-0.0064-.h5\n",
      "Epoch 95/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0066\n",
      "Epoch 96/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0066\n",
      "\n",
      "Epoch 00096: loss did not improve from 0.00636\n",
      "\n",
      "Epoch 00096: ReduceLROnPlateau reducing learning rate to 3.200000264769187e-07.\n",
      "Epoch 97/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0066\n",
      "Epoch 98/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0065\n",
      "\n",
      "Epoch 00098: loss did not improve from 0.00636\n",
      "\n",
      "Epoch 00098: ReduceLROnPlateau reducing learning rate to 6.400000529538374e-08.\n",
      "Epoch 99/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0065\n",
      "Epoch 100/100\n",
      "20000/20000 [==============================] - 27s 1ms/step - loss: 0.0065\n",
      "\n",
      "Epoch 00100: loss did not improve from 0.00636\n",
      "\n",
      "Epoch 00100: ReduceLROnPlateau reducing learning rate to 1.2800001059076749e-08.\n"
     ]
    }
   ],
   "source": [
    "#训练模型\n",
    "from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau\n",
    "filepath = main_path + \"modles/W-\" + \"-{epoch:3d}-{loss:.4f}-.h5\"\n",
    "checkpoint = ModelCheckpoint(filepath,\n",
    "                    monitor='loss',\n",
    "                    verbose=1,\n",
    "                    save_best_only=True,\n",
    "                    mode='min',\n",
    "                    period=2,\n",
    "                    save_weights_only=True\n",
    "                    )\n",
    "reduce_lr = ReduceLROnPlateau(monitor='loss', \n",
    "                    factor=0.2, \n",
    "                    patience=2, \n",
    "                    verbose=1, \n",
    "                    mode='min', \n",
    "                    min_delta=0.0001, \n",
    "                    cooldown=0, \n",
    "                    min_lr=0\n",
    "                    )\n",
    "callbacks_list = [checkpoint, reduce_lr]\n",
    "model.fit(\n",
    "    x=[np.array(encode_input[:20000]), np.array(decode_input[:20000])],\n",
    "    y=np.array(decode_output[:20000]),\n",
    "    epochs=100,\n",
    "    batch_size=64, \n",
    "    verbose=1,\n",
    "    callbacks=callbacks_list, \n",
    "    # class_weight=None, \n",
    "    # max_queue_size=5, \n",
    "#    workers=1, \n",
    "#    use_multiprocessing=False,\n",
    "    # shuffle=False,\n",
    "#    initial_epoch=initial_epoch_\n",
    "    )\n",
    "# model.save(main_path+'modles/model.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 8343,
     "status": "ok",
     "timestamp": 1588309168109,
     "user": {
      "displayName": "Jayee Wong",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhdF_nXQgSlUtrprqDLXzf9Kn59RlonvtGF4nHZ=s64",
      "userId": "08268369137892915441"
     },
     "user_tz": -480
    },
    "id": "DIPdi5jJ6To3",
    "outputId": "fb4ff73c-2b7b-409d-88fa-ef1a5581b159"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "#加载模型\n",
    "model.load_weights('model/W-- 40-0.0563-.h5')\n",
    "target_token_dict_inv = {v: k for k, v in target_token_dict.items()}\n",
    "print('Done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 155461,
     "status": "ok",
     "timestamp": 1588309337115,
     "user": {
      "displayName": "Jayee Wong",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14GhdF_nXQgSlUtrprqDLXzf9Kn59RlonvtGF4nHZ=s64",
      "userId": "08268369137892915441"
     },
     "user_tz": -480
    },
    "id": "waCFmuVtjRn5",
    "outputId": "4f15a6a7-e0cb-4857-dcaa-8759a706d86d"
   },
   "outputs": [],
   "source": [
    "from keras.preprocessing import sequence\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib\n",
    "import jieba\n",
    "import requests\n",
    "\n",
    "def get_input(seq):\n",
    "    seq = ' '.join(jieba.lcut(seq, cut_all=False))\n",
    "    # seq = ' '.join(seq)\n",
    "    seq = seq.split(' ')\n",
    "    print(seq)\n",
    "    seq = ['<START>'] + seq + ['<END>']\n",
    "    seq = seq + ['<PAD>'] * (34 - len(seq))\n",
    "    print(seq)\n",
    "    for x in seq:\n",
    "        try:\n",
    "            source_token_dict[x]\n",
    "        except KeyError:\n",
    "            flag=False\n",
    "            break\n",
    "        else:\n",
    "            flag=True\n",
    "    if(flag):\n",
    "        seq = [source_token_dict[x] for x in seq]\n",
    "    return flag, seq\n",
    "def get_ans(seq):\n",
    "    decoded = decode(\n",
    "    model,\n",
    "    [seq],\n",
    "    start_token=target_token_dict['<START>'],\n",
    "    end_token=target_token_dict['<END>'],\n",
    "    pad_token=target_token_dict['<PAD>'],\n",
    "    # top_k=10,\n",
    "    # temperature=1.0,\n",
    "  )\n",
    "    print(' '.join(map(lambda x: target_token_dict_inv[x], decoded[0][1:-1])))\n",
    "\n",
    "while True:\n",
    "    seq = input()\n",
    "    if seq == 'x':\n",
    "        break\n",
    "    flag, seq = get_input(seq)\n",
    "    if(flag):\n",
    "        get_ans(seq)\n",
    "    else:\n",
    "        print('听不懂呢。')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "GlYO_CK9tLYE"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "train.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
